﻿using JumbotOA.DBUtility;
using log4net;
using log4net.Repository;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Configuration.Json;
using Microsoft.Extensions.DependencyInjection;
using SimpleCrawler.Crawler;
using SimpleCrawler.Crawler.WebCrawler;
using SimpleCrawler.DAL;
using SimpleCrawler.DependencyInjection;
using SimpleCrawler.Entity;
using SimpleCrawler.Extensions;
using SimpleCrawler.Logger;
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;

namespace SimpleCrawler
{
    class Program
    {
        public static void Main(string[] args)
        {
            //基础单位 一个站点一个IP只开一个线程访问 
            List<CrawlerAction> listCrawler = new List<CrawlerAction>()
            {
                //new CrawlerAction(new GcxxCrawler()),
                //new CrawlerAction(new TYCCrawler())
                //new CrawlerAction(new SitemapAndBaiduUpdate())
                //new CrawlerAction(new TianYanCrawler())
                //new CrawlerAction(new ZhiHuBySouGouCrawler()),

                new CrawlerAction(new SouGouCrawler()),
                new CrawlerAction(new QiHu360WenDaCrawler()),
                new CrawlerAction(new SinaIAskCrawler()),
                new CrawlerAction(new BaiDuWDCrawler()),
                new CrawlerAction(new ZhiHuCrawler()),
                new CrawlerAction(new TuBaTuCrawler()),
            };

            #region 
            //爬虫编码
            //1001,"跑项目"
            //1002, "天眼查"
            //1003, "天眼查"
            //-1, "百度咨询推广"
            #endregion
            #region 编号
            //1004, "搜狗问问"
            //1005, "360问答"
            //1006, "知乎搜狗"
            #endregion
            #region 从来源中获取任务,并将任务赋给对应的爬虫
            //test
            List<CrawlerTask> tasklist =new List<CrawlerTask>() {
                //new CrawlerTask { CrawlerID = 1001, TaskName = "招标信息抓取" },
                //new CrawlerTask { CrawlerID = 1002, TaskName = "公司信息抓取",NeedNext=true,StartPage=1,EndPage=2 },
                //new CrawlerTask { CrawlerID = 1003, TaskName = "招标信息抓取" }
                //new CrawlerTask { CrawlerID = 1006, TaskName = "知乎问答抓取",KeyWord="一级建筑师",NeedNext=true,StartPage=1,EndPage=2},

                 new CrawlerTask { CrawlerID = 1004, TaskName = "搜狗问答抓取",KeyWord="一级建筑师",NeedNext=true,StartPage=1},
                 new CrawlerTask { CrawlerID = 1005, TaskName = "360问答抓取",KeyWord="一级建筑师",NeedNext=true,StartPage=1},
                 new CrawlerTask { CrawlerID = 1007, TaskName = "Sina爱问抓取",KeyWord="一级建筑师",NeedNext=true,StartPage=1,EndPage=2},
                new CrawlerTask { CrawlerID = 1008, TaskName = "百度问答抓取",KeyWord="一级建筑师",NeedNext=true,StartPage=1},
                 new CrawlerTask { CrawlerID = 1009, TaskName = "知乎问答抓取",KeyWord="一级建筑师",Total=3},
                 new CrawlerTask { CrawlerID = 1010, TaskName = "土巴兔问答抓取",KeyWord="一级建筑师",NeedNext=true,StartPage=1,EndPage=1},
            };
            #endregion
            foreach (var c in listCrawler)
            {
                for (int i = tasklist.Count - 1; i >= 0; i--)
                {
                    if (c.CrawlerID == tasklist[i].CrawlerID)
                    {
                        c.TaskList.Add(tasklist[i]);
                        tasklist.Remove(tasklist[i]);
                    }
                }
            }

            Parallel.ForEach(listCrawler,new ParallelOptions { MaxDegreeOfParallelism=10}, (c, state) =>
             {
                 if(c.TaskList.Count>0)
                    c.Start();
             });

            Console.Read();//正式环境注释  结束后直接退出
        }

        /// <summary>
        /// 注入服务
        /// </summary>
        /// <param name="services"></param>
        public void ConfigureServices(IServiceCollection services)
        {
            //配置文件
            ConfigurationBuilder builder = new ConfigurationBuilder();
            builder.AddJsonFile("appsettings.json");
            IConfiguration configuration = builder.Build();
            services.AddSingleton<IConfiguration>(configuration);
            //ADO操作
            string connectionString = configuration.GetSection("ConnectionStrings")["SqlServerConnection"];
            DbHelperSQLP DbHelperSQL = new DbHelperSQLP(connectionString);
            CrawlerDAL crawlerDAL = new CrawlerDAL(DbHelperSQL);
            services.AddSingleton<CrawlerDAL>(crawlerDAL);
            //url过滤服务注入
            services.AddBloomFilterServiceCollectionExtensions();
            //注入gbk gb2312编码
            System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
        }
    }
}
